"""
https://js.langchain.com/docs/modules/data_connection/document_transformers/#get-started-with-text-splitters
"""

from PyCmpltrtok.common import sep
from langchain.text_splitter import RecursiveCharacterTextSplitter
import os

if 0:
    text_id = 'js_example'
    text = '''Hi.\n\nI'm Harrison.\n\nHow? Are? You?\nOkay then f f f f.\
    This is a weird text to write, but gotta test the splittingggg some how.\n\n\
    Bye!\n\n-H.'''

if 0:
    text_id = 'my'
    text = '''1Hi\n\n2Haha\n\n3Yi\n\n4Yes\n\n5No\n\n6Hai\n\n7Iee\n\n'''

text_id = 'my_long'
text = '''1Hi\n\n21Haha\n22long\n23longer\n24longest\n25this is very long\n\n3Yi\n\n4Yes\n\n5No\n\n6Hai\n\n7Iee\n\n'''

sep()
print(text)
sep()


def split(path, text, chunk_size=10, chunk_overlap=1):
    splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
    output = splitter.create_documents([text])
    # print(output)
    variable4check = [(i, obj.page_content) for i, obj in enumerate(output)]
    print(f'Writing to {path}')
    with open(path, 'w', encoding='utf8', newline='\r\n') as f:
        for xtuple in variable4check:
            f.write(f'<{xtuple[0]}>{xtuple[1]}</{xtuple[0]}>' + '\n')
    print(f'Done: {path}')


if __name__ == '__main__':

    def main():
        dir = os.path.dirname(os.path.abspath(__file__))
        dir = os.path.join(dir, 'split_experiment.tmp.d')
        os.makedirs(dir, exist_ok=True)
        for size in range(15, 20 + 1):
            for i in range(5, min(size, 8)):
                filename = os.path.join(dir, f'{text_id}-size-{size}-overlap-{i}.tmp.txt')
                print(filename)
                split(filename, text, chunk_size=size, chunk_overlap=i)

    main()
